# -*- coding: utf-8 -*-
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from pyquery import PyQuery as pq
import re
from scrapy.http import Request as sreq
from girl.items.city import *
from scrapy.shell import inspect_response
import time
import datetime
import pymongo
client = pymongo.MongoClient('127.0.0.1')
db = client.city3
class CitySpider(CrawlSpider):
    name = "cityccc"
    domain = "https://www.cityheaven.net"
    custom_settings = {
        # "CONCURRENT_REQUESTS":95,
        "HTTPCACHE_ENABLED" : False
    }

    def start_requests(self):
        urls = db.stores.find({},{"storesUrl":1})
        for x in urls:
            yield sreq(x.get("storesUrl"),callback=self.store)

    def store(self,response):
        attendance = []
        for x in response.xpath("//*[@class='shukkin-list-container']/div"):
            attendance.append(x.xpath("a/@href").extract())
        item["attendance"] = attendance
        item["date"] = datetime.datetime.date()
        yield item